package com.example.springboot_demo.service;

import com.alibaba.fastjson.JSON;
import com.example.springboot_demo.mappers.TecDocumentMapper;
import com.example.springboot_demo.pojo.DevDocument;
import com.example.springboot_demo.pojo.TecDocument;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;

@Component
public class TecService {

    @Autowired
    TecDocumentMapper tecDocumentMapper;

    List<String> userAgent = Arrays.asList("Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;AvantBrowser)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;360SE)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;TheWorld)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;TencentTraveler4.0)",
            "Opera/9.80(Macintosh;IntelMacOSX10.6.8;U;en)Presto/2.8.131Version/11.11",
            "Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
            "Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.1;Trident/5.0",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36",
            "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
            "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
            "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
            "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
            "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
            "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
            "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
            "PostmanRuntime/7.28.4",
            "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52");


    public Set<String> getFromDb() {
        List<String> strings = tecDocumentMapper.queryIds();
        return new HashSet<>(strings);
    }


    public List<Map<String, Object>> getPage(String keyWord, int year) {
        String start = year + "-01-01";
        String end = year + "-12-31";
        // 构建请求参数
        Map<Object, Object> params = new HashMap<>();
        params.put("websiteid", "110000000000000");
        params.put("scope", "basic");
        params.put("q", keyWord);
        params.put("pg", "10");
        params.put("cateid", "54");
        params.put("pos", "title_text,titlepy,infocontent,filenumbername,keyword,contentdescribe");
        params.put("begin", start);
        params.put("end", end);
        params.put("dateField", "deploytime");
        params.put("selectFields", "title,content,deploytime,_index,url,cdate,keyword,contentdescribe,sectitle,picpath,columnname,themename,publishgroupname,publishtime,metaid,bexxgk,columnid,infocontenthashcode");
        params.put("group", "distinct");
        params.put("highlightConfigs", "[{\"field\":\"infocontent\",\"numberOfFragments\":2,\"fragmentOffset\":0,\"fragmentSize\":110,\"noMatchSize\":110}]");
        params.put("highlightFields", "title_text,infocontent,webid");
        params.put("level", "6");
        params.put("sortFields", "[{\"name\":\"extend1\",\"type\":\"desc\"},{\"name\":\"jsearch_score\",\"type\":\"desc\"}]");
        params.put("hidCol", "fbafd13557fa453d9c59432567d2b150");
        Set<String> fromDb = getFromDb();
        for (int page = 1;;page++) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            params.put("p", page);
            StringBuilder urlBuilder = new StringBuilder("https://www.miit.gov.cn/search-front-server/api/search/info?");
            for (Map.Entry<Object, Object> entry : params.entrySet()) {
                if (urlBuilder.length() > 60) {
                    urlBuilder.append("&");
                }
                try {
                    urlBuilder.append(URLEncoder.encode(entry.getKey().toString(), "UTF-8"))
                            .append("=")
                            .append(URLEncoder.encode(entry.getValue().toString(), "UTF-8"));
                } catch (UnsupportedEncodingException e) {
                    throw new RuntimeException(e);
                }
            }
            //创建get请求对象
            CloseableHttpClient client = HttpClients.createDefault();
            HttpGet get = new HttpGet(urlBuilder.toString());
            get.setHeader("Connection", "keep-alive");
            get.setHeader("sec-ch-ua", "\" Not;A Brand\";v=\"99\", \"Microsoft Edge\";v=\"91\", \"Chromium\";v=\"91\"");
            get.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
            get.setHeader("X-Requested-With", "XMLHttpRequest");
            get.setHeader("sec-ch-ua-mobile", "?0");
            get.setHeader("Content-Type", "application/x-www-form-urlencoded");
            get.setHeader("Sec-Fetch-Site", "same-origin");
            get.setHeader("Sec-Fetch-Mode", "cors");
            get.setHeader("Sec-Fetch-Dest", "empty");
            get.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6");
            get.addHeader("User-Agent", userAgent.get(RandomUtils.nextInt(0, userAgent.size() - 1)));
            String resultString = "";

            try {

                //启动执行请求，并获得返回值
                CloseableHttpResponse response = client.execute(get);
                //得到返回的entity对象
                HttpEntity resBody = response.getEntity();
                //把实体对象转换为string
                resultString = EntityUtils.toString(resBody, "UTF-8");
            } catch (Exception e) {
                e.printStackTrace();
                continue;
            }
            Map resMap = null;
            try {
                resMap = JSON.parseObject(resultString, Map.class);
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println(resultString);
            }
            boolean flag = (boolean) resMap.get("success");
            if (!flag) {
                System.out.println(resMap);
                System.exit(0);
            }
            Map<String, Object> data = (Map<String, Object>) resMap.get("data");
            Map<String, Object> searchResult = (Map<String, Object>) data.get("searchResult");
            if (searchResult == null) {
                break;
            }
            List<Map<String, Object>> list = (List<Map<String, Object>>) searchResult.get("dataResults");
            if (CollectionUtils.isEmpty(list)) {
                break;
            }
            Integer total = (Integer) searchResult.get("total");
            int maxPage = (total + 9) / 10;

            System.out.println("当前年度：" + year + ", 关键词：" + keyWord + ", 总条数:" + total + ", 总页数：" + maxPage + ", 当前页数：" + page);
            for (Map<String, Object> item : list) {
                List<Map<String, Object>> groupData = (List<Map<String, Object>>) item.get("groupData");
                if (CollectionUtils.isEmpty(groupData)) {
                    continue;
                }
                Map<String, Object> stringObjectMap = groupData.get(0);
                Map<String, Object>  realItem = (Map<String, Object>) stringObjectMap.get("data");
                if (fromDb.contains((String) realItem.get("url"))) {
                    continue;
                }
                TecDocument content = getContent(realItem);
                //插入数据库
                if (content != null) {
                    try {
                        tecDocumentMapper.insert(content);
                        System.out.println("插入成功:" + content.getTitle());
                    } catch (Exception e) {
                        e.printStackTrace();
                        System.out.println("插入失败"+content);
                    }
                }
            }
            if (page >= maxPage) {
                break;
            }

        }
        return null;
    }

    public TecDocument getContent(Map<String, Object> map) {
        HashMap<String, String> heards = new HashMap<>();
        //添加请求头参数
        heards.put("User-Agent", "userAgent.get(RandomUtils.nextInt(0, userAgent.size() - 1))");
        heards.put("Accept-Encoding", "gzip, deflate, br");
        heards.put("Connection", "keep-alive");
        heards.put("Accept", "application/json, text/plain, */*");
        heards.put("Accept-Encoding", "gzip, deflate, br");

        Document document = null;
        TecDocument govDocument = new TecDocument();
        govDocument.setUrl((String) map.get("url"));

        govDocument.setTitle((String) map.get("title"));
        govDocument.setDeploytime((String) map.get("deploytime"));
        govDocument.setColumnname((String) map.get("columnname"));

        try {
            document = Jsoup.connect(govDocument.getUrl()).headers(heards).get();
            // 来源
            Elements ent = document.select("div.xxgk-box span");
            for (int i = 0; i < ent.size(); i++) {
                String text = ent.get(i).text();
                if (i == 1) {
                    govDocument.setSector(text);
                } else if (i == 5) {
                    govDocument.setFileNumber(text);
                }  else if (i == 7) {
                    govDocument.setCtime(text);
                }else if (i == 11) {
                    govDocument.setInstitution(text);
                } else if (i == 13) {
                    govDocument.setCategory(text);
                }
            }
            //文章内容处理
            Element content = document.select("div.ccontent.center").first();
            if (content != null) {
                String text = content.text();
                StringBuilder stringBuilder = new StringBuilder();
                if (text.contains("兼并")) {
                    stringBuilder.append("兼并,");
                } else {
                    stringBuilder.append("--,");
                }
                if (text.contains("收购")) {
                    stringBuilder.append("收购,");
                } else {
                    stringBuilder.append("--,");
                }
                if (text.contains("并购")) {
                    stringBuilder.append("并购,");
                } else {
                    stringBuilder.append("--,");
                }
                if (text.contains("重组")) {
                    stringBuilder.append("重组");
                } else {
                    stringBuilder.append("--");
                }
                String tag = stringBuilder.toString();
                if (StringUtils.equals("--,--,--,--", tag)) {
                    return null;
                } else {
                    govDocument.setTag(tag);
                    govDocument.setContent("<!doctype html>\n" +
                            "<html>\n" +
                            content.outerHtml() +
                            "</html>"
                    );
                    return govDocument;
                }

            }

        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }

}
